#!/bin/bash
# docker image: train-image-cn-shanghai.cr.volces.com/train/llamafactory_mcy:cuda12.4_torch2.4_with_flash_attn_vllm_deepspeed_240828_1

model_path="/vepfs_train2/meichaoyang/model/qwen/Qwen2-7B-Instruct"


export WANDB_API_KEY=9afc62359e50f5d0b24fee88ce7ce8d162e998ed
export WANDB_DISABLED=true


echo "配置环境变量"
export NCCL_IB_DISABLE=0
export NCCL_NET_GDR_LEVEL=2
export NCCL_IB_QPS_PER_CONNECTION=4
export NCCL_IB_TC=160
export NCCL_IB_TIMEOUT=22

export NCCL_DEBUG=INFO

# MASTER_IP=$(cat /etc/aistudio/masteraddr)
# apt-get install pdsh -y
# export PDSH_SSH_ARGS_APPEND=""


echo "启动训练"

MASTER_ADDR=$MLP_WORKER_0_HOST
MASTER_PORT=2222

deepspeed --hostfile=$MLP_MPI_HOSTFILE src/train.py \
    --deepspeed conf/ds_stage3_config_qwen_no_offload.json \
    --stage sft \
    --model_name_or_path $model_path \
    --do_train \
    --dataset_dir /vepfs_train2/meichaoyang/train_data \
    --dataset alpaca_zh_demo,glaive_toolcall_zh_demo \
    --preprocessing_num_workers 16 \
    --template qwen \
    --finetuning_type full \
    --output_dir /vepfs_train2/meichaoyang/checkpoint/qwen2_7b_chat_full_sft_alpaca_zh_demo_glaive_toolcall_zh_demo_bs128_lr2e-6_2node_len20k \
    --overwrite_cache \
    --per_device_train_batch_size 1 \
    --gradient_accumulation_steps 16 \
    --lr_scheduler_type cosine \
    --logging_steps 1 \
    --warmup_steps 0 \
    --save_steps 2000 \
    --save_only_model \
    --learning_rate 2e-6 \
    --num_train_epochs 3.0 \
    --plot_loss \
    --bf16 \
    --ignore_len 20480 \
    --cutoff_len 102400 \
    --flash_attn fa2 \
    --use_fast_tokenizer true
